import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class URLExtractor {

    public List<String> extract(URL root){

        List<URL>  result = new ArrayList<URL>();
        List<String> extracted = new ArrayList<String>();

        URL home = null;

        try {
            home = new URL(root.toString());
        } catch (MalformedURLException e) {
           e.printStackTrace();
        }

        try {

            URLConnection conn = null;
            if (home != null) {
                conn = home.openConnection();
            }

            String type = null;
            if (conn != null) {
                type = conn.getContentType();
            }

            if (type != null) {
                if(type.toLowerCase().contains("text") && type.toLowerCase().contains("html")){

                    InputStream is = conn.getInputStream();
                    BufferedReader reader = new BufferedReader(new InputStreamReader(is));

                    String s;
                    StringBuilder sb = new StringBuilder();
                    while ((s = reader.readLine()) != null){
                        sb.append(s);
                    }

                    Pattern p = Pattern.compile("(href=\")([\\p{Graph}]*)(\")");

                    Matcher m = p.matcher(sb.toString());
                    while(m.find()){
                        String extracted_URL = m.group(2).split("[?]")[0];

                        if (!extracted.contains(extracted_URL)){
                            extracted.add(extracted_URL);

                        }
                    }
                    extracted = prepareURL(extracted, home.toString());

                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        }

        return extracted;
    }

    private List<String> prepareURL(List<String> links, String root_URL){
        List<String> result = new ArrayList<String>();
        for (String url : links){
            String good_URL;
            if (url.startsWith("#") || (url.startsWith("/#"))){
                continue;
            } else {
                if (url.startsWith("http://") || url.startsWith("https://")) {
                    if (url.contains("#")) {
                        url = (url.split("#"))[0];
                    }
                    good_URL = url;
                } else if (url.startsWith("/")) {
                    if (url.contains("#")) {
                        url = (url.split("#"))[0];
                    }
                    good_URL = root_URL + url;
                } else {
                    continue;
                }
            }
            if (!result.contains(good_URL)){
                result.add(good_URL);
            }

        }
        return result;
    }

}
